import numpy as np
import pandas as pd
from num2words import num2words

# sklearn
from sklearn import metrics
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.metrics import mean_squared_error, mean_absolute_error

# Tensorflow
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers

# Visualisation libraries

## Text
from colorama import Fore, Back, Style
from IPython.display import Image, display, Markdown, Latex, clear_output

## progressbar
import progressbar

## plotly
from plotly.offline import init_notebook_mode, iplot 
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px

## seaborn
import seaborn as sns

## matplotlib
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse, Polygon
from matplotlib.font_manager import FontProperties
import matplotlib.colors as mcolors
from matplotlib import cm
plt.style.use('seaborn-whitegrid')
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
plt.rcParams['text.color'] = 'k'
%matplotlib inline

import warnings
warnings.filterwarnings("ignore")

Home / Deep Learning

TensorFlow - Keras

• Keras Multi-layer Perceptron (MLP) for Binary Classifications
• Keras Multi-layer Perceptron (MLP) for Multi-Class Classifications
• Keras Multi-layer Perceptron (MLP) for Image Classifications
• Keras Multi-layer Perceptron (MLP) for Regression

Machine learning can be categorized as supervised, semi-supervised, or unsupervised learning methods. Deep learning can be supervised or unsupervised, and it is based on artificial neural networks (ANN) with representation learning. In this article, we demonstrate implementing a deep neural network (DNN) in Keras.

Deep Neural Networks (DNNs), which are also known as convolutional networks, are comprised of several layers of nonlinear operations. The goal of deep learning methods is to learn feature hierarchies. Higher levels of features are formed by combining lower-level features [1].

Dataset

a random n-class classification dataset can be generated using sklearn.datasets.make_classification. Here, we generate a dataset with two features and 1000 instances. Moreover, the dataset is generated for multiclass classification with five classes.

n_features =2
n_classes = 4
X, y = make_classification(n_samples = int((n_classes-1)*1e3), n_features = n_features, n_redundant=0, n_classes = n_classes,
                           n_informative=2, random_state=1, n_clusters_per_class=1)
Labels_dict = dict(zip(list(np.unique(y)), [num2words(x).title() for x in np.unique(y)]))

Data = pd.DataFrame(data = X, columns = ['Feature %i' % (i+1) for i in range(n_features)])
Target = 'Outcome Variable'
Data[Target] = y
display(Data)

def Plot_Data(X, y, PD, ax = None):
    # adding margins
    lims = float('%1.e' % X.min())-PD['pad'], float('%1.e' % X.max())+PD['pad']   
    # Figure
    if ax == None:
        fig, ax = plt.subplots(1, 1, figsize=(PD['FigSize'], PD['FigSize']))

    scatter = ax.scatter(X[:,0], X[:,1], s=PD['cricle_size'],
                         c=y, edgecolor = 'Navy', alpha = PD['alpha'], cmap = PD['ColorMap'])
    _ = ax.legend(handles=scatter.legend_elements()[0], labels= PD['Labels'],
                  fancybox=True, framealpha=1, shadow=True, borderpad=PD['BP'], loc='best', fontsize = 14)
    _ = ax.set_xlim(lims)
    _ = ax.set_ylim(lims)
    _ = ax.set_xlabel('Feature 1')
    _ = ax.set_ylabel('Feature 2')
    _ = ax.set_aspect(1)
    _ = ax.grid(PD['grid'])
    
PD = dict(BP = .5, alpha=.7, bg_alpha = 0.25, grid = True, cricle_size = 50,
          FigSize = 7, h=0.02, pad=1, ColorMap =  'tab10', Labels = list(Labels_dict.values()))

Plot_Data(X, y, PD = PD, ax = None)

	Feature 1	Feature 2	Outcome Variable
0	-1.689380	1.233636	2
1	-1.357824	1.236826	2
2	1.104089	1.308663	3
3	-1.395940	-0.465010	0
4	-0.221240	-2.495895	0
...	...	...	...
2995	-1.457264	1.179276	2
2996	0.998499	1.209994	3
2997	-0.301584	-2.114714	0
2998	1.111228	0.011635	3
2999	-1.316420	-0.448300	0

3000 rows × 3 columns

Train and Test sets

def DatasetTargetDist(Inp, Target, Labels_dict, PD):
    # Table
    Table = Inp[Target].value_counts().to_frame('Count').reset_index(drop = False).rename(columns = {'index':Target})
    Table[Target] = Table[Target].replace(Labels_dict)
    Table['Percentage'] = np.round(100*(Table['Count']/Table['Count'].sum()),2)
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=PD['column_widths'],
                        specs=[[{"type": "table"},{"type": "pie"}]])

    # Right
    fig.add_trace(go.Pie(labels=Table[Target].values, values=Table['Count'].values,
                         pull=PD['pull'], textfont=dict(size= PD['textfont']),
                         marker=dict(colors = PD['PieColors'], line=dict(color='black', width=1))), row=1, col=2)
    fig.update_traces(hole=PD['hole'])
    fig.update_layout(height = PD['height'], legend=dict(orientation="v"), legend_title_text= PD['legend_title'])
    
    # Left
    T = Table.copy()
    T['Percentage'] = T['Percentage'].map(lambda x: '%%%.2f' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)

    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= PD['TableColors'][0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = PD['tablecolumnwidth'],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [PD['TableColors'][1], PD['TableColors'][1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + Target + '<b>', 'x':PD['title_x'],
                                     'y':PD['title_y'], 'xanchor': 'center', 'yanchor': 'top'})
    fig.show()

Pull = [.01 for x in range((len(Labels_dict)-1))]
Pull.append(.1)
PD = dict(PieColors = px.colors.sequential.deep, TableColors = ['Navy','White'], hole = .4,
          column_widths=[0.6, 0.4],textfont = 14, height = 350, tablecolumnwidth = [0.25, 0.15, 0.15],
          pull = Pull, legend_title = Target, title_x = 0.5, title_y = 0.8)
del Pull
DatasetTargetDist(Data, Target, Labels_dict, PD)

StratifiedKFold is a variation of k-fold which returns stratified folds: each set contains approximately the same percentage of samples of each target class as the complete set.

Test_Size = 0.3
sss = StratifiedShuffleSplit(n_splits=1, test_size=Test_Size, random_state=42)
_ = sss.get_n_splits(X, y)
for train_index, test_index in sss.split(X, y):
    # X
    if isinstance(X, pd.DataFrame):
        X_train, X_test = X.loc[train_index], X.loc[test_index]
    else:
        X_train, X_test = X[train_index], X[test_index]
    # y    
    if isinstance(y, pd.Series):
        y_train, y_test = y[train_index], y[test_index]
    else:
        y_train, y_test = y[train_index], y[test_index]
del sss

def Train_Test_Dist(X_train, y_train, X_test, y_test, PD, Labels_dict = Labels_dict):
    def ToSeries(x):
        if not isinstance(x, pd.Series):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=3, horizontal_spacing = 0.02, column_widths= PD['column_widths'],
                        specs=[[{"type": "table"},{'type':'domain'}, {'type':'domain'}]])
    # Right
    C = 2
    for y in [ToSeries(y_train).replace(Labels_dict), ToSeries(y_test).replace(Labels_dict)]:
        fig.add_trace(go.Pie(labels= list(Labels_dict.values()),
                             values= y.value_counts().values, pull=PD['pull'],
                             textfont=dict(size=PD['textfont']),
                             marker=dict(colors = PD['PieColors'],
                             line=dict(color='black', width=1))), row=1, col=C)
        fig.update_traces(hole=.5)
        fig.update_layout(legend=dict(orientation="v"), legend_title_text= PD['legend_title'])
        C+=1
    
    # Left
    
    # Table
    Table = pd.DataFrame(data={'Set':['X_train','X_test','y_train','y_test'],
                       'Shape':[X_train.shape, X_test.shape, y_train.shape, y_test.shape]}).astype(str)
    T = Table.copy()
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
        
    TableColors = PD['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = PD['tablecolumnwidth'],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)
    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':PD['title_x'],
                                     'y':PD['title_y'], 'xanchor': 'center', 'yanchor': 'top'})
    if not PD['height'] == None:
        fig.update_layout(height = PD['height'])
    fig.show()
    
PD.update(dict(column_widths=[0.3, 0.3, 0.3], tablecolumnwidth = [0.2, 0.4], height = 350, legend_title = Target))

Train_Test_Dist(X_train, y_train, X_test, y_test, PD)
#
y_train = keras.utils.to_categorical(y_train, num_classes=len(Labels_dict))
y_test = keras.utils.to_categorical(y_test, num_classes=len(Labels_dict))

Modeling: PyTorch Multi-layer Perceptron (MLP) for Binary Classification

A multi-layer perceptron (MLP) is a class of feedforward artificial neural networks (ANN). scikit-learn.org has a well-written article regarding MLP and interested readers are encouraged to see this article.

Moreover, in this article, we present a multi-class MLP using Keras and focus on implementing this method in Keras. We define our model by using Sequential class. Moreover, we consider the rectified linear unit) (ReLU) as the activation function. An activation function allows for complex relationships in the data to be learned. For the last year, we use the softmax function, also known as softargmax or normalized exponential function.

model = keras.Sequential(name = 'Multi_Class_MLP')
model.add(layers.Dense(64, input_dim = X.shape[1], activation='relu', name='Layer1'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(64, activation='relu', name='Layer2'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(len(Labels_dict), activation='softmax', name='Layer3'))
model.summary()
tf.keras.utils.plot_model(model, show_shapes=True, show_layer_names=True, expand_nested = True, rankdir = 'LR')

Model: "Multi_Class_MLP"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
Layer1 (Dense)               (None, 64)                192       
_________________________________________________________________
dropout (Dropout)            (None, 64)                0         
_________________________________________________________________
Layer2 (Dense)               (None, 64)                4160      
_________________________________________________________________
dropout_1 (Dropout)          (None, 64)                0         
_________________________________________________________________
Layer3 (Dense)               (None, 4)                 260       
=================================================================
Total params: 4,612
Trainable params: 4,612
Non-trainable params: 0
_________________________________________________________________

# Number of iterations
IT = int(1e3)+1

model.compile(optimizer= keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True),
              loss='categorical_crossentropy', metrics=['accuracy', tf.keras.metrics.Recall()])

# Train model
history = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs= IT, batch_size=128, verbose = 0)

def Search_List(Key, List): return [s for s in List if Key in s]

Metrics_Names = {'loss':'Loss', 'accuracy':'Accuracy', 'mae':'MAE', 'mse':'MSE', 'recall': 'Recall'}

def Table_modify(df, Metrics_Names = Metrics_Names):
    df = df.rename(columns = Metrics_Names)
    df = df.reindex(sorted(df.columns), axis=1)
    df.insert(loc = 0, column = 'Iteration', value = np.arange(0, df.shape[0]), allow_duplicates=False)
    return df

Validation_Table = Search_List('val_',history.history.keys()) 
Train_Table = list(set( history.history.keys()) - set(Validation_Table))
Validation_Table = pd.DataFrame(np.array([history.history[x] for x in Validation_Table]).T, columns = Validation_Table)
Train_Table = pd.DataFrame(np.array([history.history[x] for x in Train_Table]).T, columns = Train_Table)
Validation_Table.columns = [x.replace('val_','') for x in Validation_Table.columns]

Train_Table = Table_modify(Train_Table)
Validation_Table = Table_modify(Validation_Table)

# Train Set Score
score = model.evaluate(X_test, y_test, batch_size=128, verbose = 0) 
score = pd.DataFrame(score, index = model.metrics_names).T
score.index = ['Train Set Score']
# Validation Set Score
Temp = model.evaluate(X_train, y_train, batch_size=128, verbose = 0) 
Temp = pd.DataFrame(Temp, index = model.metrics_names).T
Temp.index = ['Validation Set Score']
score = score.append(Temp)
score.rename(columns= Metrics_Names, inplace = True)
score = score.reindex(sorted(score.columns), axis=1)
display(score.style.set_precision(4))

	Accuracy	Loss	Recall
Train Set Score	0.9267	0.2175	0.9256
Validation Set Score	0.9348	0.1937	0.9338

def Plot_history(history, Title = False, Table_Rows = 25):
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=[0.6, 0.4],
                        specs=[[{"type": "scatter"},{"type": "table"}]])
    # Left
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Loss'].values,
                             line=dict(color='OrangeRed', width= 1.5), name = 'Loss'), 1, 1)
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Accuracy'].values,
                             line=dict(color='MidnightBlue', width= 1.5),  name = 'Accuracy'), 1, 1)
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Recall'].values,
                             line=dict(color='SeaGreen', width= 1.5), name = 'Recall'), 1, 1)
    fig.update_layout(legend=dict(x=0, y=1.1, traceorder='reversed', font_size=12),
                  dragmode='select', plot_bgcolor= 'white', height=600, hovermode='closest',
                  legend_orientation='h')
    fig.update_xaxes(range=[history.Iteration.min(), history.Iteration.max()],
                     showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    fig.update_yaxes(range=[0, 1], showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    # Right
    ind = np.linspace(0, history.shape[0], Table_Rows, endpoint = False).round(0).astype(int)
    ind = np.append(ind, history.index[-1])
    history = history[history.index.isin(ind)]
    T = history.copy()
    T[['Accuracy','Loss','Recall']] = T[['Accuracy','Loss','Recall']].applymap(lambda x: '%.4e' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    fig.add_trace(go.Table(header=dict(values = list(history.columns), line_color='darkslategray',
                                       fill_color='DimGray', align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.4, 0.4, 0.4, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray', fill=dict(color=['WhiteSmoke', 'white']),
                                      align=['center', 'center'], font_size=12,height=20)), 1, 2)
    if Title != False:
        fig.update_layout(plot_bgcolor= 'white',
                      title={'text': Title, 'x':0.46, 'y':0.94, 'xanchor': 'center', 'yanchor': 'top'},
                          yaxis_title='Frequency')
    fig.show()

Plot_history(Train_Table, Title = 'Train Set')
Plot_history(Validation_Table, Title = 'Validation Set')

def Plot_Classification(Model, X, y, PD, ax = None):
    # adding margins
    x_min, x_max = float('%1.e' % X.min())-PD['pad'], float('%1.e' % X.max())+PD['pad']
    y_min, y_max = float('%1.e' % X.min())-PD['pad'], float('%1.e' % X.max())+PD['pad']
    # Generating meshgrids
    xx, yy = np.meshgrid(np.arange(x_min, x_max, PD['h']), np.arange(y_min, y_max, PD['h']))

    # Predictions
    Pred = Model.predict(np.c_[xx.ravel(), yy.ravel()])
    if Pred.shape[1] > 1:
        Pred = Pred.argmax(axis = 1)
        
    Pred = Pred.reshape(xx.shape)
    
    # Figure
    if ax == None:
        fig, ax = plt.subplots(1, 1, figsize=(PD['FigSize'], PD['FigSize']))
        
    _ = ax.contourf(xx, yy, Pred, cmap = PD['ColorMap'], alpha=PD['bg_alpha'])

    scatter = ax.scatter(X[:,0], X[:,1], s=PD['cricle_size'],
                         c=y, edgecolor = 'Black', linewidths = 0.5, alpha = PD['alpha'], cmap = PD['ColorMap'])
    _ = ax.legend(handles=scatter.legend_elements()[0], labels= PD['Labels'],
                  fancybox=True, framealpha=1, shadow=True, borderpad=PD['BP'], loc='best', fontsize = 14)
    _ = ax.set_xlim(x_min, x_max)
    _ = ax.set_ylim(y_min, y_max)
    _ = ax.set_xlabel('Feature 1')
    _ = ax.set_ylabel('Feature 2')
    _ = ax.set_aspect(1)
    _ = ax.grid(PD['grid'])

#    
PD = dict(BP = .5, alpha=.7, bg_alpha = 0.15, grid = False, cricle_size = 50,
          FigSize = 7, h=0.02, pad=1, ColorMap =  'Set1', Labels = list(Labels_dict.values()))
fig, ax = plt.subplots(1, 2, figsize=(16, 7))
# Train Set
Plot_Classification(model, X_train, y_train.argmax(axis = 1), PD = PD, ax = ax[0])
_ = ax[0].set_title('Train Set', fontsize = 16)
# Test Set
Plot_Classification(model, X_test, y_test.argmax(axis = 1), PD = PD, ax = ax[1])
_ = ax[1].set_title('Test Set', fontsize = 16)

Confusion Matrix

The confusion matrix allows for visualization of the performance of an algorithm. Note that due to the size of data, here we don't provide a Cross-validation evaluation. In general, this type of evaluation is preferred.

def Confusion_Mat(CM_Train, CM_Test, PD, n_splits = 10):
    if n_splits == None:
        Titles = ['Train Set', 'Test Set']
    else:
        Titles = ['Train Set (CV = % i)' % n_splits, 'Test Set (CV = % i)' % n_splits]
    CM = [CM_Train, CM_Test]
    Cmap = ['Greens', 'YlGn','Blues', 'PuBu']
    for i in range(2):
        fig, ax = plt.subplots(1, 2, figsize= PD['FS'])
        fig.suptitle(Titles[i], weight = 'bold', fontsize = 16)
        _ = sns.heatmap(CM[i], annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i], ax = ax[0],
                        linewidths = 0.2, cbar_kws={"shrink": PD['shrink']})
        _ = ax[0].set_title('Confusion Matrix');
        Temp = np.round(CM[i].astype('float') / CM[i].sum(axis=1)[:, np.newaxis], 2)
        _ = sns.heatmap(Temp,
                        annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i+1], ax = ax[1],
                       linewidths = 0.4, vmin=0, vmax=1, cbar_kws={"shrink": PD['shrink']})
        _ = ax[1].set_title('Normalized Confusion Matrix');

        for a in ax:
            _ = a.set_xlabel('Predicted labels')
            _ = a.set_ylabel('True labels'); 
            _ = a.xaxis.set_ticklabels(PD['Labels'])
            _ = a.yaxis.set_ticklabels(PD['Labels'])
            _ = a.set_aspect(1)

# Train
y_pred = model.predict(X_train).argmax(axis = 1)
Reports_Train = pd.DataFrame(metrics.classification_report(y_train.argmax(axis = 1),
                                                           y_pred, target_names=list(Labels_dict.values()),
                                                           output_dict=True)).T
CM_Train = metrics.confusion_matrix(y_train.argmax(axis = 1), y_pred)
# Test
y_pred = model.predict(X_test).argmax(axis = 1)
Reports_Test = pd.DataFrame(metrics.classification_report(y_test.argmax(axis = 1),
                                                          y_pred, target_names=list(Labels_dict.values()),
                                                          output_dict=True)).T
CM_Test = metrics.confusion_matrix(y_test.argmax(axis = 1), y_pred)

Reports_Train = Reports_Train.reset_index().rename(columns ={'index': 'Train Set'})
Reports_Test = Reports_Test.reset_index().rename(columns ={'index': 'Test Set'})
                                                 
display(Reports_Train.style.hide_index().set_properties(**{'background-color': 'HoneyDew', 'color': 'Black'}).\
        set_properties(subset=['Train Set'], **{'background-color': 'SeaGreen', 'color': 'White'}))
display(Reports_Test.style.hide_index().set_properties(**{'background-color': 'Azure', 'color': 'Black'}).\
        set_properties(subset=['Test Set'], **{'background-color': 'RoyalBlue', 'color': 'White'}))

PD = dict(FS = (14, 6), annot_kws = 14, shrink = .6, Labels = list(Labels_dict.values()))
Confusion_Mat(CM_Train, CM_Test, PD = PD, n_splits = None)

Train Set	precision	recall	f1-score	support
Zero	0.930783	0.977055	0.953358	523.000000
One	0.929577	0.876660	0.902344	527.000000
Two	0.963391	0.950570	0.956938	526.000000
Three	0.915888	0.935115	0.925401	524.000000
accuracy	0.934762	0.934762	0.934762	0.934762
macro avg	0.934910	0.934850	0.934510	2100.000000
weighted avg	0.934931	0.934762	0.934477	2100.000000

Test Set	precision	recall	f1-score	support
Zero	0.943478	0.968750	0.955947	224.000000
One	0.898148	0.858407	0.877828	226.000000
Two	0.959641	0.951111	0.955357	225.000000
Three	0.904762	0.928889	0.916667	225.000000
accuracy	0.926667	0.926667	0.926667	0.926667
macro avg	0.926507	0.926789	0.926450	900.000000
weighted avg	0.926457	0.926667	0.926363	900.000000

---

References

1. S. Jothilakshmi, V.N. Gudivada, in Handbook of Statistics, 2016
2. Keras developer guides
3. Multilayer perceptron wikipedia page
4. Confusion matrix wikipedia page

---